原始数据¶
导入¶
In [218]:
# df = pd.read_excel(r'../ML/ML计算/特征过滤法.xlsx',sheet_name='0')
# df=pd.read_csv(r'small_loan.csv')
# df.drop('id', inplace=True, axis=1)
df = pd.read_csv(r'../01ML_Case/house_price/train.csv')
df.drop('Id', inplace=True, axis=1)
df.shape
Out[218]:
In [219]:
df.head()
Out[219]:
report 1st¶
In [220]:
dqrs=DQReport(data=df, target='Y').SReport()
dqrs
# pde.dftohtml(dqrs)
Out[220]:
In [221]:
report_view(dqrs)
dfc_str,dfc_num,dfc_num_int,dfc_num_float=dfc_type(df,if_print=True)
初始处理¶
ytype¶
In [222]:
# ytype='fenlei'#分类
ytype='huigui'#回归
Y (类型记得转化)¶
In [223]:
df.rename(columns={'SalePrice':'Y'},inplace=True)
# df.rename(columns={'response':'Y'},inplace=True)
# df['Y'] = df['Y'].map({"YES": 1, "NO": 0})
In [224]:
if ytype=='fenlei':
df['Y']=df['Y'].astype(int)
else:
df['Y']=df['Y'].astype(float)
In [225]:
if ytype=='fenlei':
# 分类
dfc_str_col_plt_count(df,'Y')
else:
# 回归
print("Skewness: %f" % df['Y'].skew())
print("Kurtosis: %f" % df['Y'].kurt())
dfc_num_plt_huigui('Y',df)
dfc_num_col_plt_value(df)
# Ycut
df['Ycut']=pd.qcut(df.Y, 4)
df['Ycut']=df['Ycut'].astype(str)
print(df.Ycut.value_counts())
wvf符号替换为nan¶
In [226]:
df = wrong_value_fillna(wrong_value=list(wvf_txt())).fit_transform(df)
-----column type转换¶
In [227]:
# df['age']=df['age'].astype(float)
# df['income']=df['income'].astype(float)
# df['children']=df['children'].astype(str)
# df_fill_na['age']=df_fill_na['age'].astype(float)
# df_fill_na['income']=df_fill_na['income'].astype(float)
In [228]:
# float可能为int的
dfc_type_mcol_float_may_int(dfc_num_float,df)
Out[228]:
In [229]:
dfc_str,dfc_num,dfc_num_int,dfc_num_float=dfc_type(df,if_print=True)
-----手动¶
In [230]:
# df['children']=df['children'].astype(str)
空值填充 df_fill_na¶
xgb_fill¶
In [231]:
# df_fill_na=xgb_fill().fit_transform(df.drop(columns=['Y']),df.Y) #无Y列出错
-----其他fill¶
In [232]:
# df['age'].fillna(df['age'].mode()[0],inplace=True)
# df['age']=df['age'].astype(int)
null列分析¶
In [233]:
dfc_null(df,0.05)
Out[233]:
-----null列删除¶
In [234]:
df.drop(columns=dfc_null(df,0.2).index,inplace=True)
df.shape
Out[234]:
null列再次查看¶
In [235]:
df.isnull().sum()[df.isnull().sum()>0]
Out[235]:
report 2nd¶
In [236]:
dqrs=DQReport(data=df, target='Y').SReport()
dqrs
# pde.dftohtml(dqrs)
Out[236]:
最终确认dfc list¶
In [237]:
report_view(dqrs)
dfc_str,dfc_num,dfc_num_int,dfc_num_float=dfc_type(df,if_print=True)
异常值¶
发现¶
In [238]:
abnormal_value(df,dfc_num,dqrs)#print_value_list:bool=False
Out[238]:
-----填充¶
In [239]:
# df = fix_outlier(dfc_num,how='quartile').fit_transform(df) #dfc_num注意使用，有时候不需要全部数值列都处理
# df = fix_outlier([]).fit_transform(df) #自定义列表
# df = fix_outlier().fit_transform(df)
再确认¶
In [240]:
# abnormal_value(df,dfc_num,dqrs)
变量分析¶
-----dfc_cycle¶
In [241]:
dfc_cycle_index=-1
# dfc_cycle_list=dfc_num+dfc_str
dfc_cycle_list=dfc_str+dfc_num
def dfc_cycle (dfc_cycle_list:list,df:pd.DataFrame,dqrs:pd.DataFrame):
global dfc_cycle_index
dfc_cycle_index+=1
if dfc_cycle_index < len(dfc_cycle_list):
if ytype=='fenlei':
if df.dtypes[dfc_cycle_list[dfc_cycle_index]]=='object':
print(dfc_str_plt_fenlei_huigui(dfc_cycle_list[dfc_cycle_index],df,label_col_name='Y')) #fenlei_huigui一样， label_col不一样
else:
print(abnormal_value(df,[dfc_cycle_list[dfc_cycle_index]],dqrs))
dfc_num_plt_fenlei(dfc_cycle_list[dfc_cycle_index],df) #分类
else:
if df.dtypes[dfc_cycle_list[dfc_cycle_index]]=='object':
print(dfc_str_plt_fenlei_huigui(dfc_cycle_list[dfc_cycle_index],df,label_col_name='Ycut'))#fenlei_huigui一样， label_col不一样
else:
print(abnormal_value(df,[dfc_cycle_list[dfc_cycle_index]],dqrs))
dfc_num_plt_huigui(dfc_cycle_list[dfc_cycle_index],df) #回归
else:
print('dfc_cycle complete. dfc_cycle_index:',dfc_cycle_index)
dfc_cycle_index=0
# dfc_cycle_list
In [242]:
dfc_cycle(dfc_cycle_list,df,dqrs)
In [243]:
# for x in df.x3.unique():
# s1=df[df.x3==x]['Y']
# print(x)
# print(s1.mean(),s1.std())
卡方 类别 vs 分类¶
In [244]:
if ytype=='fenlei':
fea_imp_list_kf_str_fenlei=fea_imp_kf_str_fenlei(df)
else:
# 回归
fea_imp_list_kf_str_fenlei=fea_imp_kf_str_fenlei(df,'Ycut')
anova¶
In [245]:
if ytype=='fenlei':
# 数值 vs 分类
fea_imp_list_num_ttest_anova_fenlei=fea_imp_num_ttest_anova_fenlei(df,dfc_num,'Y',fig_size=(8,8))
print(fea_imp_list_num_ttest_anova_fenlei)
else:
# 类别 vs 回归
fea_imp_list_str_ttest_anova_huigui=fea_imp_str_ttest_anova_huigui(df,dfc_str,fig_size=(8,8))
print(fea_imp_list_str_ttest_anova_huigui)
皮尔逊系数 数值 vs 回归¶
In [246]:
if ytype=='fenlei':
dfcorr=dfc_plt_corr(df,fig_size=(10,10),label_col_name=dfc_num[0])
else:
dfcorr=dfc_plt_corr(df,fig_size=(16,16))
In [247]:
dfcorr_largest(dfcorr,0.7)
Out[247]:
In [248]:
# s=dfcorr[dfcorr>0.5].Y.dropna()
# s.drop(index='Y')
# s1=dfcorr[dfcorr>0.5].MSSubClass.dropna()
# s1=pd.DataFrame(s1.drop(index='MSSubClass'))
# s1
斯皮尔曼系数 数值 vs 回归¶
In [249]:
def spearman(frame, features,label_col_name='Y'):
spr = pd.DataFrame()
spr['feature'] = features
#Signature: a.corr(other, method='pearson', min_periods=None)
#Docstring:
#Compute correlation with `other` Series, excluding missing values
# 计算特征和 SalePrice的 斯皮尔曼 相关系数
spr['spearman'] = [frame[f].corr(frame[label_col_name], 'spearman') for f in features]
spr = spr.sort_values('spearman')
plt.figure(figsize=(6, 0.25*len(features))) # width, height
sns.barplot(data=spr, y='feature', x='spearman', orient='h')
spearman(df,dfc_num)
线性关系可视化 数值 vs 回归¶
In [250]:
dfc_num_plt_huigui_gmap_pairplot(df,dfc_num)
report 最终¶
In [251]:
DQReport(data=df, target='Y').SReport()
Out[251]:
In [252]:
report_view(dqrs)
dfc_str,dfc_num,dfc_num_int,dfc_num_float=dfc_type(df,if_print=True)
dftm¶
In [308]:
dftm=df.copy()
try:
dftm=dftm.drop(['Ycut'],axis=1)
except:
pass
dftm.head()
Out[308]:
xgb填空¶
In [309]:
dftm=xgb_fill().fit_transform(dftm,dftm.Y) #无Y列出错
null列 dftm查看¶
In [310]:
# dfc_null(dftm,0.001)
# dftm.isnull().sum().max()
dftm.isnull().sum().sum()
Out[310]:
-----str列处理, 空值填充¶
In [311]:
# dfc_type(dftm,True,False)[0]
In [312]:
# dftm['Y']=dftm['Y'].astype(int)
# dftm['Y'].value_counts()
# df['Y'] = np.where(df['Y'] == 'YES', 1, 0) #如果Y列不是0,1,2类型则记得转换
dftm['Embarked'].fillna(dftm['Embarked'].mode()[0],inplace=True)
DQReport(data=dftm, target='Y').SReport()
report dftm¶
In [313]:
dqrs=DQReport(data=dftm, target='Y').SReport()
In [314]:
report_view(dqrs)
dfc_str,dfc_num,dfc_num_int,dfc_num_float=dfc_type(df,if_print=True)
dftm_fea¶
In [315]:
dftm_fea=dftm.loc[:,['Y']+['BsmtQual', 'WoodDeckSF', 'HeatingQC', 'Foundation', 'SaleCondition', 'BsmtExposure', 'MSSubClass', 'Condition1', '1stFlrSF', 'RoofStyle', 'BldgType', 'LotShape', 'TotalBsmtSF', 'YearRemodAdd', 'ExterQual', 'RoofMatl', 'MasVnrArea', 'OverallQual', 'Neighborhood', 'GarageArea', 'BsmtFinSF1', 'GarageYrBlt', 'SaleType', 'HouseStyle', 'LotArea', '2ndFlrSF', 'MSZoning', 'CentralAir', 'YearBuilt', 'BsmtFinType1', 'LotFrontage', 'YrSold', 'ExterCond', 'BsmtUnfSF', 'KitchenQual', 'OpenPorchSF', 'PavedDrive', 'Exterior1st', 'Exterior2nd', 'LotConfig', 'MoSold', 'LandContour', 'MasVnrType', 'GrLivArea', 'OverallCond', 'BsmtCond']]
get_dummies¶
In [316]:
dftm=df_get_dummies(dftm)
# dftm.head()
dftm_fea=df_get_dummies(dftm_fea)
----train test split¶
In [317]:
x_train=dftm.drop(['Y'],axis=1)
try:
x_train=dftm.drop(['Ycut'],axis=1) # 回归前面有可能有Ycut
except:
pass
y_train=dftm['Y']
x_train_fea=dftm_fea.drop(['Y'],axis=1)
try:
x_train_fea=dftm_fea.drop(['Ycut'],axis=1) # 回归前面有可能有Ycut
except:
pass
y_train_fea=dftm_fea['Y']
特征重要性 xgb tm_plt¶
In [318]:
# 分类
# tm=xgb.XGBClassifier().fit(x_train, y_train)
# 回归
tm=xgb.XGBRegressor().fit(x_train, y_train)
In [319]:
tm_plt=tm_plt_importance(tm)
In [320]:
fea_imp_list_xgb=tm_plt_importance_list(tm_plt)
-----特征最终筛选¶
In [321]:
top=20
s1=fea_imp_list_xgb[:top]
s2=fea_imp_list_kf_str_fenlei[0][:top]
if ytype=='fenlei':
s3=list(fea_imp_list_num_ttest_anova_fenlei.loc['feature',:])
else:
s3=list(fea_imp_list_str_ttest_anova_huigui.loc['feature',:])
s3=s3[:top]
In [322]:
print('xgb: ',len(s1))
print(s1)
print('')
if ytype=='fenlei':
print('kf, str: ',len(s2))
else:
print('kf, str. !!!回归的Ycut仅供参考: ',len(s2))
print(s2)
print('')
if ytype=='fenlei':
print('anova, num. 用于分类: ',len(s3))
print(s3)
else:
print('anova, str. 用于回归:', len(s3))
print(s3)
print('')
dfc_str,dfc_num,dfc_num_int,dfc_num_float=dfc_type(df,if_print=True)
# print('重要的str')
# print(list(set(tpil_top10).intersection(dfc_str)))
# print('不重要的str')
# print(list(set(dfc_str).difference(tpil)))
# print('重要的num')
# print(list(set(tpil_top10).intersection(dfc_num)))
# print('不重要的num')
# print(list(set(dfc_num).difference(tpil)))
In [323]:
# xgb &卡方
s4=set(s1).intersection(s2)
print(s4)
In [324]:
# xgb & anova
s5=set(s1).intersection(s3)
print(s5)
In [325]:
# xgb & dfc_num
s6=set(s1).intersection(dfc_num)
print(len(s6))
print(s6)
In [326]:
s5.difference(s4)
Out[326]:
In [327]:
set(s2).difference(s3)
Out[327]:
In [328]:
set(s3).difference(s2)
Out[328]:
In [329]:
set(s2).intersection(s3)
Out[329]:
In [330]:
s=list(set(s1+s2+s3))
print(len(s))
print(s)
有无筛选特征 模型对比¶
In [332]:
from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test = train_test_split(x_train,y_train,test_size=0.25,random_state=0)
x_train_fea,x_test_fea,y_train_fea,y_test_fea = train_test_split(x_train_fea,y_train_fea,test_size=0.25,random_state=0)
In [333]:
print(x_train.shape)
print(len(y_train))
print(x_test.shape)
print(len(y_test))
print(x_train_fea.shape)
print(len(y_train_fea))
print(x_test_fea.shape)
print(len(y_test_fea))
In [334]:
from sklearn.preprocessing import StandardScaler
#标准化，返回值为标准化后的数据
std_x = StandardScaler()
x_train = std_x.fit_transform(x_train)
x_test = std_x.transform(x_test)
x_train_fea = std_x.fit_transform(x_train_fea)
x_test_fea = std_x.transform(x_test_fea)
In [335]:
#线性回归
from sklearn import linear_model
from sklearn.linear_model import LinearRegression,Ridge,Lasso
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import GridSearchCV
线性回归_无筛选特征¶
In [353]:
#gridsearch
params={'alpha':[0.001,0.01,0.1,1,10,100]}#粗调节
tm = Ridge()
tm=GridSearchCV(tm,params,cv=10)
tm.fit(x_train,y_train)
tm.best_params_
Out[353]:
In [354]:
tm_names=[]
tm_scores_train=[]
tm_scores_test=[]
def tm_result_list(tm_name,tm_model):
tm_names.append(tm_name)
tm_scores_train.append(tm_model.score(x_train,y_train))
tm_scores_test.append(tm_model.score(x_test,y_test))
In [355]:
tm_result_list('线性回归_无筛选特征',tm)
In [356]:
pd.DataFrame({'model':tm_names,'train':tm_scores_train,'test':tm_scores_test})
Out[356]:
线性回归_有筛选特征¶
In [357]:
params={'alpha':[0.001,0.01,0.1,1,10,100]}#粗调节
tm_fea= Ridge()
tm_fea=GridSearchCV(tm_fea,params,cv=10)
tm_fea.fit(x_train_fea,y_train_fea)
tm_fea.best_params_
Out[357]:
In [358]:
tm_names.append('线性回归_有筛选特征')
tm_scores_train.append(tm_fea.score(x_train_fea,y_train_fea))
tm_scores_test.append(tm_fea.score(x_test_fea,y_test_fea))
In [359]:
pd.DataFrame({'model':tm_names,'train':tm_scores_train,'test':tm_scores_test})
Out[359]:
xgboost_无筛选特征¶
In [360]:
tm = xgb.XGBRFRegressor()
params = {
'n_estimators':range(100,200,50),
'max_depth':range(2,15,4),
'learning_rate':np.linspace(0.01,3,5),
}
tm=GridSearchCV(tm,param_grid=params,cv=5)
tm.fit(x_train, y_train)
tm.best_params_
Out[360]:
In [361]:
tm_result_list('xgboost_无筛选特征',tm)
In [362]:
pd.DataFrame({'model':tm_names,'train':tm_scores_train,'test':tm_scores_test})
Out[362]:
xgboost_有筛选特征¶
In [363]:
tm_fea = xgb.XGBRFRegressor()
params = {
'n_estimators':range(100,200,50),
'max_depth':range(2,15,4),
'learning_rate':np.linspace(0.01,3,5),
}
tm_fea=GridSearchCV(tm_fea,param_grid=params,cv=5)
tm_fea.fit(x_train_fea, y_train_fea)
tm_fea.best_params_
Out[363]:
In [364]:
tm_names.append('xgboost_有筛选特征')
tm_scores_train.append(tm_fea.score(x_train_fea,y_train_fea))
tm_scores_test.append(tm_fea.score(x_test_fea,y_test_fea))
只是筛选掉无用特征，其他条件一样的情况下，模型泛化能力提升。
- 线性回归提升8%，效果理想（线性回归对模型特征较敏感）。
- Xgboost提升不明显（Xgboost的基分类器决策树对无用特征不敏感）¶
In [365]:
pd.DataFrame({'model':tm_names,'train':tm_scores_train,'test':tm_scores_test})
Out[365]: